package com.itcast.spark.baseCount

import org.apache.spark.mllib.linalg
import org.apache.spark.mllib.random.RandomRDDs
import org.apache.spark.mllib.linalg.{Matrix, Vectors}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.SparkSession
import org.apache.spark.mllib.stat.{MultivariateStatisticalSummary, Statistics}
import org.apache.spark.rdd.RDD

/**
 * DESC:统计特性
 * 这里实现的是基于mllib的数据统计
 * 最大值
 * 最小值
 * 非0值个数
 * 均值
 * 方差
 * 标准差
 */
object _01SparkMSummary {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setAppName("_04libsvmSparkSQL").setMaster("local[*]")
    val spark: SparkSession = SparkSession.builder().config(conf).getOrCreate()
    val sc: SparkContext = spark.sparkContext
    //如何读取vector的类型的数据转化为RDD[Vector]
    val data: RDD[linalg.Vector] = sc.textFile("./datasets/mldata/testSummary.txt")
      .map(x => x.toDouble)
      .map(x => Vectors.dense(x))
    //调用API的统计特性
    val summary: MultivariateStatisticalSummary = Statistics.colStats(data)
    println(summary.numNonzeros)
    println(summary.mean)
    println(summary.max)
    println(summary.min)
    println(summary.variance)
    println(summary.count)
    val value1: RDD[Double] = sc.parallelize(Array(1.0, 2.0, 3.0))
    val value2: RDD[Double] = sc.parallelize(Array(2.0, 4.0, 6.0))
    val corr1: Double = Statistics.corr(value1, value2)
    //pearson皮尔逊相关系数--------概率论相关系数=协方差./标准差
    //pearson相似度是在cos余弦相似度的基础上做了一些改进，对数据做了标准
    println("correlation value is:", corr1) //如何计算的？(correlation value is:,1.0)
    val value3: RDD[linalg.Vector] = sc.parallelize(Seq(
      Vectors.dense(1.0, 2.0, 3.0),
      Vectors.dense(2.0, 4.0, 6.0),
      Vectors.dense(7.0, 8.0, 9.0)
    ))
    val matrix: Matrix = Statistics.corr(value3)
    println(matrix)
    // from the standard normal distribution.  一般情况现实世界的分部都是正态分布
//    val data: RDD[Double] = RandomRDDs.normalRDD(sc, 100L, seed = 123L)

    val dataSamples: RDD[Int] = sc.parallelize(1 to 10)
    dataSamples.sample(true,0.7,123L).foreach(println(_))
    val ints: Array[Int] = dataSamples.takeSample(true, 7, 123L)
    println(ints)
  }
}
